package no.ntnu.idi.deid.preprocessor;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Scanner;

import no.ntnu.idi.deid.config.FilePaths;
import no.ntnu.idi.deid.documentunits.Document;
import no.ntnu.idi.deid.preprocessor.postagger.POStagger;
import no.ntnu.idi.deid.preprocessor.sentenizer.externaltools.DocumentToDocumentHandler;
import no.ntnu.idi.deid.preprocessor.wordcount.WordCountManager;

import org.annolab.tt4j.TreeTaggerException;

public class Documents implements Iterable<Document> {

	private String inputFilesFolder = FilePaths.getInputPath();
	private String sentenizedFilesFolder = FilePaths.getSentenizedFilesPath();
	private String posTagFilesFolder = FilePaths.getPOSTaggedFilesPath();
	private String washedFilesFolder = FilePaths.getWashedFilesFolder();

	private List<Document> documents = new ArrayList<Document>();
	
	private WordCountManager wcm;

	// Make sentenceFiles for each file
	// For each sentenceFile: make documents

	public Documents(String inputFolder, String sentenizedFolder,
			String posTagfolder) throws FileNotFoundException, IOException,
			TreeTaggerException {
		inputFilesFolder = inputFolder;
		sentenizedFilesFolder = sentenizedFolder;
		posTagFilesFolder = posTagfolder;
		process();

	}
	
	public void clearFolders() {
		deleteAllFilesInFolder(posTagFilesFolder);
		deleteAllFilesInFolder(sentenizedFilesFolder);
		deleteAllFilesInFolder(FilePaths.getOutputFolderPath());
		deleteAllFilesInFolder(washedFilesFolder);
		deleteAllFilesInFolder(FilePaths.getDeIdentifiedFilesFolderPath());
	}
	
	
	public Documents() throws FileNotFoundException, IOException, TreeTaggerException  {
		process();

	}

	public void sentenizeInputFiles() throws FileNotFoundException,
			IOException, TreeTaggerException {
		DocumentToDocumentHandler dh = new DocumentToDocumentHandler();
		dh.processDocuments(washedFilesFolder, sentenizedFilesFolder, "NO",
				"prepVocabulary.txt", "abbreviations.txt", "UTF-8");
	}

	public void process() throws FileNotFoundException, IOException, TreeTaggerException {
		clearFolders();
		washAllDocuments();
		sentenizeInputFiles();
		posTagInputFiles();
		makeDocuments();
		countWords();
	}

	private void countWords() {
		List<Map<String, Integer>> wordCountList = new ArrayList<Map<String, Integer>>();
		for (Document doc : documents) {
			wordCountList.add(doc.getWordCountManager().getMap());
		}
		wcm = new WordCountManager(wordCountList);
		wcm.generateStatsDocument(FilePaths.getWordCountStatsFilePath());
	}
	
	public void makeDocuments() throws IOException, TreeTaggerException {
		File[] filesFolder = (new File(sentenizedFilesFolder)).listFiles();
		File[] postagFilesFolder = (new File(posTagFilesFolder)).listFiles();
		Arrays.sort(filesFolder);
		Arrays.sort(postagFilesFolder);
		if (filesFolder.length == postagFilesFolder.length) {
			for (int i = 0; i < postagFilesFolder.length; i++) {
				if ((filesFolder[i].getName() + ".tag")
						.equals(postagFilesFolder[i].getName())) {
					Document d = new Document(i,filesFolder[i].getPath(),postagFilesFolder[i].getAbsolutePath());
					documents.add(d);
				} else {
					System.err
							.println("filename and postagFilename don't match");
				}
			}
		} else {
			System.err
					.println("filesFolder.length and postagFilesFolder.length don't match");
		}
	}

	private void deleteAllFilesInFolder(File file) {
		if(file.isDirectory()) {
			File[] filesFolder = file.listFiles();
			for(File f: filesFolder) {
				f.delete();
			}
		}
	}
	
	private void washAllDocuments(){
		File[] inputFiles = (new File(inputFilesFolder)).listFiles();
		for(File file: inputFiles){
			wash(file);
		}
	}
	
	//Removes characters like: (vasketinput) 13,160,65279
	private void wash(File file){
		Scanner sc = null;
		Writer w = null;
		try {
			w = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(FilePaths.getWashedFilesFolder() + file.getName()),"utf-8"));
			sc = new Scanner(file);
			String line;
			while(sc.hasNext()){
				line = sc.nextLine();
				line.replaceAll( Character.toString((char)13), " ");
				line.replaceAll( Character.toString((char)160), "");
				line.replaceAll( Character.toString((char)65279), "");
				line.replaceAll("\t"," ");
//				line.replaceAll(System.getProperty("line.separator"), " ");
//				line.replaceAll("[^a-zA-Z_0-9]", "");
				//|\\\\|\\/|\\.|\\(|\\)|\\:|\\;|\\<|\\>|\\-|\\,|\\_
				if(checkLine(line)){
					w.write(line + "\n");
				}
				else {
					w.write("\n");
				}
				
			}
		} catch (IOException e) {
			e.printStackTrace();
		} finally{
			sc.close();
			try {
				w.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}
	
	private boolean checkLine(String line){
		for (int i = 0; i < line.length(); i++) {
			if(Character.isLetterOrDigit(line.charAt(i))){return true;}
		}
		return false;
	}
	
	private void deleteAllFilesInFolder(String file) {
		deleteAllFilesInFolder(new File(file));
	}
	
	public void posTagInputFiles() throws IOException, TreeTaggerException {
		POStagger.tagFolder(sentenizedFilesFolder, posTagFilesFolder);
	}

	public List<Document> getDocuments() {
		return documents;
	}

	public WordCountManager getWordCountManager() {
		return wcm;
	}

	@Override
	public Iterator<Document> iterator() {
		return documents.iterator();
	}
	
}
